from sklearn.feature_extraction import DictVectorizer
import datetime
from sklearn import metrics
from ArtMAP.models import ArtMAP, AdaptiveResonance
from collections import Counter, defaultdict
import matplotlib.pyplot as plt
import json
import gzip
from sklearn.cross_validation import train_test_split
from sklearn.naive_bayes import MultinomialNB
__author__ = 'pstalidis'

start_time = datetime.datetime.now()
# path1 = "/media/pstalidis/eecf9b83-9877-4f85-94b9-b0a4d07ade6b/home/radius/Downloads/ICT4Growth/ARTmap/"
# path2 = "/media/pstalidis/eecf9b83-9877-4f85-94b9-b0a4d07ade6b/home/radius/Downloads/ICT4Growth/BOOK_TF/"
path1 = "/home/panagiotis/Downloads/ICT4Growth/subset/"
path2 = "/home/panagiotis/Downloads/ICT4Growth/subset/BOOK_TF/"

# read data from files
user_ids = json.loads(open(path1+"userids.json").readline())
book_ids = json.loads(open(path1+"bookids.json").readline())
score_vectors = open(path1+"scorevectors.csv")
bookvec = zip(book_ids, [line.strip().split("\t") for line in score_vectors])

reversed_scores = defaultdict(Counter)
# mlabels = [0, 1, 2, 3, 4, 5]
mlabels = ["UNK", "NEG", "POS"]
for b in xrange(0, len(book_ids)):
    for u in xrange(0, len(user_ids)):
        # reversed_scores[book_ids[b]][user_ids[u]] = int(bookvec[b][1][u])
        if int(bookvec[b][1][u]) > 3:
            reversed_scores[book_ids[b]][user_ids[u]] = "POS"
        # elif int(bookvec[b][1][u]) == 0:
        #     reversed_scores[book_ids[b]][user_ids[u]] = "UNK"
        else:
            reversed_scores[book_ids[b]][user_ids[u]] = "NEG"

user_descriptions = defaultdict(dict)
for book in reversed_scores.keys():
    for user in reversed_scores[book]:
        user_descriptions[user][book] = reversed_scores[book][user]

book_descriptions = dict()
for book in book_ids:
    book_descriptions[book] = json.loads(gzip.open(path2+book+".json.gz", 'rb').readline())

# hide some books
train_book_ids, test_book_ids = train_test_split(reversed_scores.keys(), test_size=0.3, random_state=0)

train_scores = defaultdict(Counter)
for b in train_book_ids:
    for u in reversed_scores[b].keys():
        train_scores[u][b] = reversed_scores[b][u]

test_scores = defaultdict(Counter)
for b in test_book_ids:
    for u in reversed_scores[b].keys():
        test_scores[u][b] = reversed_scores[b][u]


train_user_dictionary = []
train_book_dictionary = []
train_ratings = []
test_user_dictionary = []
test_book_dictionary = []
test_ratings = []

for userid in train_scores:
    for bookid in train_scores[userid]:
        train_user_dictionary.append(user_descriptions[userid])
        train_book_dictionary.append(book_descriptions[bookid])
        train_ratings.append(train_scores[userid][bookid])

for userid in test_scores:
    for bookid in test_scores[userid]:
        test_user_dictionary.append(user_descriptions[userid])
        test_book_dictionary.append(book_descriptions[bookid])
        test_ratings.append(test_scores[userid][bookid])

book_transform = DictVectorizer()
user_transform = DictVectorizer()

book_descriptions = train_book_dictionary + test_book_dictionary
book_transform.fit(book_descriptions)
user_transform.fit(train_user_dictionary)
train_book_vectors = book_transform.transform(train_book_dictionary)
train_user_vectors = user_transform.transform(train_user_dictionary)
test_user_vectors = user_transform.transform(test_user_dictionary)
test_book_vectors = book_transform.transform(test_book_dictionary)

print datetime.datetime.now() - start_time
uth = 0.7
bth = 0.9

t0 = datetime.datetime.now()
ucls = AdaptiveResonance(shrink_threshold=uth)
bcls = AdaptiveResonance(shrink_threshold=bth)
clf = MultinomialNB()

# from sklearn.linear_model import SGDClassifier
# lcf = SGDClassifier(loss='hinge', penalty='l2', alpha=1e-3, n_iter=5)

art = ArtMAP(cluster_algorithm_for_first_class=ucls, cluster_algorithm_for_second_class=bcls, classifier=clf)
art.multi_fit(X=train_user_vectors, Y=train_book_vectors, z=train_ratings)
# art.fit(X=train_user_vectors, Y=train_book_vectors, z=train_ratings)

print datetime.datetime.now() - t0

predicted = art.multi_predict(X=test_user_vectors, Y=test_book_vectors)
# predicted = art.predict(X=test_user_vectors, Y=test_book_vectors)

cm = metrics.confusion_matrix(y_true=test_ratings, y_pred=predicted, labels=mlabels)
print "total results", cm
plt.matshow(cm)
plt.title('Confusion matrix')
plt.colorbar()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.show()